########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
"""Utility functions
====================
codeifing additional structure on the data-base
"""
import copy
from obj.flat import stagvalue
from obj.obj import Obj,db
from obj.dt import DT
from model import *
from tagsyntax import *
from lib.quoteinterface import getFXrate
from modellib import *
from trnlib import *

def fix_money_currency():
    pass

            
def holding_source_cmp_date(hs1,hs2):
    """ Compare two sources for order of importantce according to their *computed* date range
You must compute  _DTSTART_ ,_DTEND_  attribute in hs1 and hs2 before using this method.
"""
    if hs1._DTSTART_[0]!=hs2._DTSTART_[0]:
        # older sources are better
        return cmp(hs1._DTSTART_[0],hs2._DTSTART_[0])
    else:
        # if two sources have same date, the one that covers more is better 
        return -cmp(hs1._DTEND_[0],hs2._DTEND_[0])

def holding_source_cmp(hs1,hs2):
    """ Compare two sources for order of importantce.
Return 1 if hs2 is more important and -1 if hs1 is more important.
You must compute  _DTSTART_ ,_DTEND_ and _FULL_ attribute
in hs1 and hs2 before using this method.
"""
    # source without a date range are the worst
    if not hs2._DTSTART_: return -1
    elif not hs1._DTSTART_: return 1
    # full source are the best
    elif hs1.full() and hs2.full():
        return holding_source_cmp_date(hs1,hs2)
    elif hs1.full(): return -1
    elif hs2.full(): return 1
    # otherwise use dates
    else: return holding_source_cmp_date(hs1,hs2)

def get_sorted_hs(holding):
    """Return list of holding-sources used by holding except for _MERGED_,
sorted in order of importance for merger of transactions.
While doing this, add computed information to the holding-source object:
_FULL_ - True if the source contains all transactions for its date range
_DTSTART_, _DTEND_ - Date range of source, either taken from source or computed from range on transactions dates
    """
    holding_sources=db.getallobjects(HoldingSource,{'Holding':[holding],'!=FILE':['_MERGED_']})

    for hs in holding_sources:
        if len(hs._FULL_): continue # Skip if already compuetd _FULL_,_DTSTART_, _DTEND_
        full = dtstart = dtend = []
        if hs.FULL:
            full = hs.FULL
            dtstart = hs.DTSTART
            dtend = hs.DTEND
        if not (full and dtstart and dtend):
            accsrc=db.getobjectfromline(AccountSource,{'FILE':hs.FILE,'Account':holding.Account})
            if accsrc:
                if accsrc.FULL and not full: full =accsrc.FULL
                if accsrc.DTSTART and not dtstart: dtstart = accsrc.DTSTART
                if accsrc.DTEND and not dtend:  dtend = accsrc.DTEND
        if not (full and dtstart and dtend):
            sc=db.getallobjects('SourceClass',{'FILE':hs.FILE})
            if sc:
                sc=sc[0]
                if not full:      full=sc.FULL
                if not dtstart: dtstart=sc.DTSTART
                if not dtend:  dtend=sc.DTEND
        if not full: full =[False]
        
        # If not defined externally, a date range of a full source is the range
        # of all dates that appears in its transactions.
        if full[0] and not (dtstart and dtend):
            transactions=db.getallobjects(Transaction,{'HoldingSource':[hs]})
            if transactions:
                dttrade=[t.DTTRADE[0] for t in transactions if t.DTTRADE]
                dtsettles=[t.DTSETTLE[0] for t in transactions]  # DTSETTLE are mandatory so dont check for its existance
                dt=dtsettles+dttrade
                if not dtstart:
                    dtstart=[DTSTART(copy.copy(min(dt)))]
                    dtstart[0].startofday()
                if not dtend:
                    dtend=[DTEND(copy.copy(max(dt)))]
                    dtend[0].endofday()
        
        hs._FULL_= full
        hs._DTSTART_= dtstart
        hs._DTEND_=dtend

    holding_sources.sort(holding_source_cmp)
    return holding_sources


def do_mhst(self,line={}):
    """Add to DB merged history information.
You must call mt before this command."""
    for h in getholdings(line):
        hs=getholdingsources(h) # Get merged HoldingSource
        if not hs: continue      
        # Generate a Statement line from each merged transaction
        for t in db.getallobjects(Transaction,{'HoldingSource':hs}):
            if not (t.UNITPRICE and t.UNITPRICE[0]): continue
            for a in h.Asset:
                line={'FILE':['_MERGED_'],
                    'Asset':[a],
                    'UNITPRICE':t.UNITPRICE,
                    'DTASOF':t.DTTRADE if t.DTTRADE else t.DTSETTLE}
                db.line_update(line)

def merge_statements(holding):
    """Merge all statements of a single holding from all sources and store them in DB
    as new statements from the source _MERGED_
    Note you must call merge_transactions first
    Note that you can perform this operation, for a given holding, only once after building the DB.
    """
    hs=getholdingsources(holding) # Get merged HoldingSource
    if not hs:
        pwarn('No merged transactions',holding)
        return

    # TODO: Generate zero statement for COMPLETE or FULL sources that have nothing on holding
    # TODO: Merge all existing Statement
        
    # Generate a Statement line from each merged transaction
    line=None
    units=0
    transactions=db.getallobjects(Transaction,{'HoldingSource':hs})
    transactions.sort(key=lambda t: t.DTSETTLE[0])
    for t in transactions:
        
        if not t.UNITS: continue
        if not t.UNITS[0]: continue
        dtasof=copy.deepcopy(t['DTSETTLE'][0])
        dtasof.endofday()
        # update only once a day
        if line and line['DTASOF'][0]!=dtasof:
            db.line_update(line)
            line=None
        units+=t.UNITS[0]
        line={'FILE':['_MERGED_'],
              'Holding':[holding],
              'UNITS':[UNITS(units)],
              'DTASOF':[dtasof]}
    if line: db.line_update(line)


def do_ms(self,line={}):
    """Add to DB merged statements.
You must call mt before this command."""
    holdings=getholdings(line)
    for i,h in enumerate(holdings):
        print i
        merge_statements(h)

def mtphase1(hs,transactions,fullhs,mtrans,overlap_range=10):
    """Phase1: merge (into mtrans) transactions from full source (hs) that don't overlap (with fullhs)
including ovelrap_range days before and after each range.
return list of transactions that where not merged (otrans), and list of transactions already paired (used_mtrans)"""
    used_mtrans=[] # List of merged transactions already paired with a transaction from this source.
    otrans=[]               
    assert(hs._DTSTART_)
    assert(hs._DTEND_)
    for t in transactions:
        date=t.DTSETTLE[0]
        if not (hs._DTSTART_[0]<=date and date<=hs._DTEND_[0]):
            pwarn('Outside range %s %s'%(str(hs._DTSTART_[0]),str(hs._DTEND_[0])),t)
            continue # Dont trust transactions outside their own range, so don't add to otrans
        if any(fhs._DTSTART_[0]-overlap_range<=date and date<=fhs._DTEND_[0]+overlap_range for fhs in fullhs):
            otrans.append(t)
            continue
        if (t.UNITS and t.UNITS[0]) or t.TOTAL or trntotal(t)!=None: #Filter out transactions that dont carry any information
            mt=trnadd(t)
            used_mtrans.append(mt)
            mtrans.append(mt)
    return otrans,used_mtrans

            
def mtphase2(otrans,mtrans,used_mtrans,fullhs,full_grace=None):
    """Phase 2 match all the other transactions from phase 1
with one of the merged transactions, and if none is found add a new merged transaction
Use greedy matching: start with the best match of original transaction to merged transaction, remove the pair and repeat."""
    if not otrans: return          
    # Build a sparse matrix of distances between every merged transaction and every other-transaction
    distmat={}
    for m in mtrans:
        # We want each transaction from this source to be used in at most one merged transaction.
        if m in used_mtrans: continue
        for o in otrans:
            dist=trndist(m,o,800)
            if dist: distmat[m,o]=dist
                
    while distmat:
        # Find best match: pair with highest score
        m1,o1=max(distmat,key=lambda k,d=distmat: d[k])
        # If the best match is not good enough there will be no other better matches.
        #if distmat[m1,o1]<800: break
        o1._MTran_=[m1]
        # All other pairing  with either m1 or o1 is now impossible so remove any such option from distmat
        mo=distmat.keys()
        for m,o in mo:
            if m==m1 or o==o1:
                del distmat[m,o]

    # Report the pairing we found and consolidate the other transactions into the merged transactions,
    # or create new merged transactions
    for o in otrans:
        level=0 # The level of merge difficulty we have
        
        if o._MTran_: # hasattr(o,'_MTran_'):
            m=o._MTran_[0]
            # parameters for fixing currency differences
            o_unitprice,o_currency=getunitpricecur(o)
            m_unitprice,m_currency=getunitpricecur(m)
            m_date=m['DTSETTLE'][0]
            
            msg=['Merging:']
            msg.append(strnprint(m))
            msg+=['with:',stagvalue('FILE',o.HoldingSource[0].FILE)]
            was_merged=False
            # Merge stags from original transaction to merged transaction
            for tag in ['DTSETTLE','DTTRADE','DTUSER',
                        'UNITS','Expense','CHECKNUM',
                        'UNITPRICE','TOTAL','FEES','INTEREST',
                        'TAXES','COST','MARKUP','MARKDOWN','LOAD','COMMISSION','WITHHOLDING','PENALTY','STATEWITHHOLDING','ACCRDINT']:
                if not o[tag]: continue
                ovalue=o[tag][0]
                # Fix currency differences
                money_tag = tag in ['UNITPRICE','TOTAL','FEES','INTEREST',
                           'TAXES','COST','MARKUP','MARKDOWN','LOAD',
                           'COMMISSION','WITHHOLDING','PENALTY',
                           'STATEWITHHOLDING','ACCRDINT']
                if money_tag:
                    if m_currency!=o_currency:
                        assert m_currency and o_currency and m_date
                        ovalue*=getFXrate(m_date,o_currency,m_currency)
                if not m[tag]:
                    level=max(level,1)
                    #msg.append('+'+stagvalue(tag,o[tag]))
                    was_merged=True
                    m.update_tag(tag,[ovalue])
                elif m[tag][0]!=o[tag][0]:
                    # This should be a single valued tag (stag) so make a warning if the new value is different,
                    if tag=='FEES':
                        level=max(level,2)
                    elif tag=='DTSETTLE': # TODO In any case you should never modify/change the DTSETTLE of m because it is part of the key.
                        def dtsettlediff(m,o,d):
                            if abs(m[tag][0]-o[tag][0])<d: return True
                            if ((not m.DTTRADE) or m.DTTRADE[0]==m.DTSETTLE[0]) and o.DTTRADE and abs(m[tag][0]-o['DTTRADE'][0])<d: return True
                            if ((not o.DTTRADE) or o.DTTRADE[0]==o.DTSETTLE[0]) and m.DTTRADE and abs(o[tag][0]-m['DTTRADE'][0])<d: return True
                            return False
                        if dtsettlediff(m,o,1):
                            level=max(level,1)
                        elif dtsettlediff(m,o,5):
                            level=max(level,2)
                        else:
                            level=max(level,3)
                    elif tag=='UNITS':
                        if abs(m[tag][0]-o[tag][0])<1:
                            level=max(level,1)
                        elif abs(m[tag][0]-o[tag][0])/max(abs(m[tag][0]),abs(o[tag][0]))<0.01:
                            level=max(level,2)
                        else:
                            level=max(level,3)
                    elif money_tag:
                        e=abs(m[tag][0]-ovalue)*2./(m[tag][0]+ovalue)
                        if e<1.005:
                            level=max(level,1)
                        elif e<1.05:
                            level=max(level,2)
                        else:
                            level=max(level,3)
                    else:
                        level=max(level,3)
                    msg.append('-'+stagvalue(tag,o[tag]))
            # Merge mtags from original transaction to merged transaction
            for tag in ['MEMO','NAME','PAYEEID','RELFITID','RELSECID']: # TODO You can't merge FITID because it is part of the key of m
                if not o[tag]: continue
                if not m[tag]:
                    level=max(level,1)
                    #msg.append('+'+stagvalue(tag,o[tag]))
                    was_merged=True
                    m.update_tag(tag,o[tag])
                else:
                    for v in o[tag]:
                        # We will merge an addtional value to the tag if it is different from all existing.
                        ok_to_merge=v not in m[tag]
                        # TODO You can't merge FITID because it is part of the key of m
                        ## Make sure the new value does not conflict with others.
                        ## A FITID tag can have more than one value only if it is from a different domain/subdomain
                        #if ok_to_merge and tag=='FITID':
                            #vd,vs,vl=v.explode()
                            #if vd:
                                #for mm in m[tag]:
                                    #md,ms,ml=mm.explode()
                                    #if md and md==vd and vs==ms: # No need to check vl!=ml
                                        #level=max(level,3)
                                        #msg.append('-'+stagvalue(tag,o[tag])) # We are merging
                                        #ok_to_merge=False
                                        #break
                        if ok_to_merge:
                            level=max(level,1)
                            was_merged=True
                            m.update_tag(tag,[v])
            # If information in o contributed in any way to m then keep a back pointer from m to o
            if was_merged:
                #m._OTran_.append(o)
                m.update_tag('_OTran_',[o])
                # FIXME You can't add NLINE and FITID directly to m because they are part of the key.
                m.update_tag('OLINE',o['NLINE'])
                m.update_tag('OFITID',o['FITID'])                
            else:
                o._MTran_=[]
        else: # We didn't found any matching MTrans for o
            date=o.DTSETTLE[0]
            # Reject probelmatic transactions
            dtstart=o.HoldingSource[0]._DTSTART_[0] if o.HoldingSource[0]._DTSTART_ else None
            if dtstart and date<dtstart:
                level=max(level,1)
                msg=['Rejecting: before '+stagvalue('DTSTART',dtstart)]
            elif any(fhs._DTSTART_[0]<=date and date<=fhs._DTEND_[0] for fhs in fullhs):
                files=[]
                for fhs in fullhs:
                    if fhs._DTSTART_[0]<=date and date<=fhs._DTEND_[0]:
                        files.append(fhs.FILE[0])
                if o.HoldingSource[0].Holding[0].ischecking() and mtrans[0].TRNTYPE[0]=='OPENING_BALANCE' and date<=mtrans[0].DTSETTLE[0]:
                    # ignore transactions that are before an opening balance
                    level=max(level,1)
                    msg=['Ignore before opening balance']
                elif full_grace and not any(fhs._DTSTART_[0]+full_grace<=date and date<=fhs._DTEND_[0]-full_grace for fhs in fullhs):
                    level=max(level,2)
                    msg=['In FULL %s but no match'%stagvalue('FILE',files)]
                    mtrans.append(trnadd(o))
                else:
                    level=max(level,3)
                    msg=['Rejecting: In FULL %s but no match'%stagvalue('FILE',files)]
            else:
                msg=['Adding']
                mtrans.append(trnadd(o))
        msg.append(strnprint(o))
        msg=' '.join(msg)
        if level>=3:
            perr(msg,o)
        elif level>1:
            pwarn(msg,o)
        elif level>0:
            pinfo(msg,o)

        
def mtphase3(mtrans,oo_filter):
    """Phase 3 try to match all OutstandingOrders to merged transactions"""
    for o in db.getallobjects('OutstandingOrder',oo_filter):
        # Skip orders that appeared in the same line with a transaction
        # all the information in them is already included in that transaction.
        if o.DTSETTLE: continue
        m=None
        for t in mtrans:
            if t.CHECKNUM and o.CHECKNUM and t.CHECKNUM[0]==o.CHECKNUM[0]:
                m=t
                break
            # TODO run other tests to see if a OO should be matched with a transaction
                  
        if not m:
            pinfo('No match',o)
            continue
        
        level=0 # The level of merge difficulty we have
        msg=['Merging:']
        msg.append(strnprint(m))
        msg+=['with:',sobj(o)]
        was_merged=False
        # Merge stags from original transaction to merged transaction
        for tag in ['DTUSER','CHECKNUM']:
            if not o[tag]: continue
            if not m[tag]:
                level=max(level,1)
                #msg.append('+'+stagvalue(tag,o[tag]))
                was_merged=True
                m.update_tag(tag,o[tag])
            elif m[tag][0]!=o[tag][0]:
                level=max(level,3)
                msg.append('-'+stagvalue(tag,o[tag]))
        # Merge mtags from original transaction to merged transaction
        for tag in ['MEMO','NAME']:
            if not o[tag]: continue
            if not m[tag]:
                level=max(level,1)
                #msg.append('+'+stagvalue(tag,o[tag]))
                was_merged=True
                m.update_tag(tag,o[tag])
            else:
                for v in o[tag]:
                    if v not in m[tag]:
                        level=max(level,1)
                        #msg.append('-'+stagvalue(tag,v))
                        was_merged=True
                        m.update_tag(tag,[v])

        if level>2: print ' '.join(msg)
        if was_merged:
            #m._OTran_.append(o)
            m.update_tag('OLINE',o[NLINE])

def do_mt(self,line={}):
    """Add to DB merged transactions.
Do this command only after ma command.
Merge all transactions of a single holding from all sources and store them in DB
as new transactions from the source _MERGED_
Note that you can perform this operation, for a given holding, only once after building the DB.
Perform this operation only after fixing all assets to have both CURRENCY and ORIGCURRENCY
"""
    if 'FULL_GRACE' not in line:
        full_grace=2
    else:
        full_grace=int(line.pop('FULL_GRACE')[0])
    transaction_filter=gettransaction_filter(line,default={})
    for i,h in enumerate(getholdings(line)):
        print i
        dtstart=DT('FUTURE')
        dtend=DT('PAST')
        complete=[True]
        mtrans=[]
        # Even if a transaction is after the range of the previouse HoldingSource, it is possible that it overlaps with one of the previouse transactions
        overlap_range=10
        fullhs=[] # All FULL holding_source up to now that have been used for the merge
        mtrans=[] # All merged transactions up to now.
        for hs in get_sorted_hs(h):
            transactions=gettransactions(transaction_filter,hs=[hs])
            if transactions:
                if any(t._MTran_ for t in transactions):
                    perr('Transaction for holding was already merged')
    
                if hs.FULL:         
                    otrans,used_mtrans=mtphase1(hs,transactions,fullhs,mtrans)
                else:
                    otrans=transactions
                    used_mtrans=[]
                mtphase2(otrans,mtrans,used_mtrans,fullhs,full_grace)
            if hs.full(): fullhs.append(hs)

        # Build a filter for Outstanding Order from the transaction filter
        oo_filter={'HoldingSource':[{'Holding':[h]}]}
        for op in ['','!=','>','>=','<','<=']:
            if op+'DTSETTLE' in transaction_filter:
                oo_filter[op+'DTUSER']=transaction_filter[op+'DTSETTLE']
        mtphase3(mtrans,oo_filter)
